from __future__ import division
import os
import sys
import shutil
import hashlib
import logging
import datetime
import smtplib
import email
import re
import subprocess
from optparse import OptionParser

#----------------------------------------------
# Send mail
def sendTextMail(mailhost,sender,recipients,subject = '',message = ''):
  try:
    mailport=smtplib.SMTP(mailhost)
  except:
    sys.stderr.write('Unable to connect to SMTP host "' + mailhost \
                       + '"!\nYou can try again later\n')
    return 0
                          
  fullmessage = 'From: ' + ' <' + sender + '> \n' +\
                'To: ' + recipients + '\n' +\
                'Subject: ' + subject + '\n' +\
                'Date: ' + email.Utils.formatdate() +'\n' +\
                '\n' +\
                message
  try:
    allrecipients = recipients.split(',')
    failed = mailport.sendmail(sender, allrecipients, fullmessage)
  except Exception as e:
    sys.stderr.write(repr(e) + "\n")
    return 0
  finally:
    mailport.quit()

  return 1

def sendMail(logFileName, notify):
  if notify != None and len(notify) > 0 :
    sender = notify.split(',')[0]
    recipients = str(notify)
    sendTextMail('localhost', sender,recipients,'Copy Failed:' + logFileName)

#-----------------------------------------------
# Logging
def log(logFileName):
  logging.basicConfig(filename = logFileName, filemode = 'a', level = logging.NOTSET, \
                      format = '%(levelname)s: %(message)s')

def verifyLog(logFile):
  logF = open(logFile,"r")
  errors = re.findall("^ERROR(.*)", logF.read(), re.M)
  logF.close()
  return errors

#-----------------------------------------------
# File And Directory
def copyFile(src,dst):
  res,error = 1,''
  if not os.path.exists(src) : 
    error = "Not exist :" + src
    return res,error
  if os.path.exists(dst): 
    res = 0
    return res,error
 
  try:
    dstDir = os.path.dirname(dst)
    if not os.path.exists(dstDir) : 
      os.makedirs(dstDir)
    shutil.copyfile(src,dst)
    res = 0
  except IOError as ioError:
    error = dst + ":" + str(ioError)
  except OSError as error:
    error = dst + ":" + str(error)
  
  return res,error 
  
def dirByHash(hashStr,length=2):
  hashLen = len(hashStr)
  dir = ""
  i = 0
  while(i < hashLen):
    if i+length < hashLen:
      dir = os.path.join(dir,hashStr[i:i+length])
    else:
      dir = os.path.join(dir,hashStr[i:])
    i += length
    
  return dir
    
def getFolderSize(folder):
  total_size = os.path.getsize(folder)
  for item in os.listdir(folder):
    itempath = os.path.join(folder, item)
    if os.path.islink(itempath):
      total_size += getSymlinkSize(itempath)
    elif os.path.isfile(itempath):
      total_size += os.path.getsize(itempath)
    elif os.path.isdir(itempath):
      total_size += getFolderSize(itempath)
  
  return total_size

def b2mg(b):
  m = b/1024/1024
  g = m/1024
  return m,g

def copySymLink(srcFile, dstFile):
  if os.path.islink(srcFile):
      try:    
        linkTo = os.readlink(srcFile)
    os.symlink(linkTo,dstFile)
      except OSError as e:
        return 1,"create symbol link failed:" + str(e)
      return 0,''
  else:
      return 1, "not symbol link"

def getSymlinkSize(symLink):
   if os.path.islink(symLink):
     try:    
       s = os.lstat(symLink).st_size
       return s
     except: 
       return 0 
   else:   
    return 0

#-----------------------------------------------
# HASH

def calHash(str, hashType = 'md5'):
  m = hashlib.md5()
  if hashType == 'sha1':
    m = hashlib.sha1()
  elif hashType == 'sha256':
    m = hashlib.sha256()
  m.update(str)
  return m.hexdigest() 
   
 
def calHashForFile(file, hashType = 'md5'):
  a_file = open(file, 'rb')
  hash = calHash(a_file.read(),hashType)
  a_file.close()
  return hash 
    
#-----------------------------------------------
# Verification
def verifyBySize(srcDir, dstDir):
  srcS = getFolderSize(srcDir)
  dstS = getFolderSize(dstDir)
 
  if srcS == dstS and dstS != 0:
    return 0,srcS,dstS
  else:
    return 1,srcS,dstS

def verifyByDiff(srcDir, dstDir):
  p = subprocess.Popen('diff -r -N ' + srcDir + ' ' + dstDir, \
                        shell=True, stdout=subprocess.PIPE,stderr=subprocess.PIPE)        
  (stdout,stderr) = p.communicate()
  if len(stdout) > 0 :
    return 1,stdout
  else:
    return 0,''

def verifyBySHA256(srcDir, dstDir):
  diffs = [] 
  for item in os.listdir(srcDir):
    srcItemPath = os.path.join(srcDir, item)
    dstItemPath = os.path.join(dstDir, item) 
    if os.path.isfile(srcItemPath) and calHashForFile(srcItemPath) != calHashForFile(dstItemPath):
      diffs.append(srcItemPath) 
    elif os.path.isdir(srcItemPath):
      res, subDiffs = verifyBySHA256(srcItemPath,dstItemPath)
      diffs.extend(subDiffs)
  return len(diffs), diffs

def verifyAll(srcDir, dstDir, logFileName):
  start = datetime.datetime.now()
  logging.info("")
  logging.info("=========================================")
  logging.info("verify start")
  logging.info("source dir:        %s" %srcDir)
  logging.info("dest dir:          %s" %dstDir)
  
  res, srcS, dstS = verifyBySize(srcDir, dstDir)
  srcSm,srcSg = b2mg(srcS)
  dstSm,dstSg = b2mg(dstS)
  logging.info("verify by size")
  logging.info("source dir size:   %d(B)=%f(MB)=%f(GB)" %(srcS,srcSm,srcSg))
  logging.info("dest dir size:     %d(B)=%f(MB)=%f(GB)" %(dstS,dstSm,dstSg))

  res2,stdout = verifyByDiff(srcDir,dstDir)
  logging.info("verify by diff")
  logging.info(stdout)

  res3, diffs = verifyBySHA256(srcDir, dstDir)
  logging.info("verify by SHA256")
  if len(diffs) == 0: 
    logging.info("")
  else: 
    for diff in diffs: logging.info(diff)

  end = datetime.datetime.now()
  logging.info("verify end")
  logging.info("total time:        " + str(end-start))
  if res or res2 or res3 : 
    logging.info("verify result:     failed")
    return 1
  else:
    logging.info("verify result:     successful")
    return 0

#-----------------------------------------------
# Main functions
def usage():
  usage = """
  usage: %prog [option]

  This Script will copy source directory to dest directory. 
  And in fact all new files will be stored in central repository directory, 
  the dest directory just are hard links.
  
  By this way we just need keep one copy for same file and save lot of disk spaces.
   
  """
  #print(usage)

def copyA(repositoryRoot, srcDir, dstDir, hash): 
  linkS = folderS = copyS = saveDiskS = 0

  if not os.path.exists(dstDir) : os.makedirs(dstDir)
  items = os.listdir(srcDir)

  for item in items:
      srcFileFP = os.path.join(srcDir,item)
      dstFileFP = os.path.join(dstDir,item)

      # if the file is link
      if os.path.islink(srcFileFP):
        copySymLink(srcFileFP,dstFileFP)
    linkS += getSymlinkSize(dstFileFP)
    logging.info("+++++++++++++++++++++++++++++++++++++++++")
    #logging.info("srcFile: %s" %srcFileFP)
    logging.info("dstFile: %s" %dstFileFP)
    logging.info("dstFile is link")
    continue

      # if the file is diretory
      if os.path.isdir(srcFileFP):
         ls,fs,cs,ss = copyA(repositoryRoot, srcFileFP, dstFileFP, hash)
     linkS += ls
     folderS +=fs
     copyS += cs
     saveDiskS += ss
     continue

      # if the file is normal file
      # copy to repository
      hashFile = dirByHash(calHashForFile(srcFileFP,hash))
      hashFileFP = os.path.join(repositoryRoot,hashFile)
      logging.info("+++++++++++++++++++++++++++++++++++++++++")
      #logging.info("srcFile: %s" %srcFileFP)
      logging.info("hashFile: %s" %hashFileFP)
 
      if not os.path.exists(hashFileFP): 
        res,error = copyFile(srcFileFP, hashFileFP)
    if res: 
      logging.error(error + " for the file " + hashFileFP) 
      continue
    else:
      hashFileS = os.path.getsize(hashFileFP)
      copyS += hashFileS
      else:
        if os.path.getsize(srcFileFP) == os.path.getsize(hashFileFP):
          logging.info("hashFile is existed in repository")
      hashFileS = os.path.getsize(hashFileFP)
      saveDiskS += hashFileS 
    else: 
      logging.info("collision: source file has different size with hash file")
      logging.info("the source will be copied to dest directly")
      logging.info("dstFile: %s" %dstFileFP)
      res,error = copyFile(srcFileFP,dstFileFP)
      if res: 
        logging.error(error + " for the file " + dstFileFP) 
      else:
        dstFileS = os.path.getsize(dstFileFP)
        copyS += dstFileS
      continue

      # create link for dest file from repository
      logging.info("dstFile: %s" %dstFileFP)
      if not os.path.exists(dstFileFP) : 
    try:
          os.link(hashFileFP,dstFileFP)
        except OSError as e:
          logging.error(str(e) + " for the file " + dstFileFP)
      else:
        logging.info("dstFile is existed") 
           
  folderS +=os.path.getsize(srcDir)

  return linkS, folderS,copyS, saveDiskS

def main(repositoryRoot,srcDir,dstDir,hash,logFileName):
  logging.info("=========================================")
  logging.info("source dir:        %s" %srcDir)
  logging.info("dest dir:          %s" %dstDir)
  logging.info("hash type:         %s" %hash)
  logging.info("start time:        %s" %str(startTime))
  logging.info("=========================================")
  logging.info("")

  linkS, folderS, copyS, saveDiskS = copyA(repositoryRoot,srcDir,dstDir,hash)
  
  logging.info("") 
  logging.info("=========================================")
  linkSm,linkSg = b2mg(linkS)
  folderSm,folderSg = b2mg(folderS)
  copySm,copySg = b2mg(copyS)
  saveDiskSm,saveDiskg = b2mg(saveDiskS)
  totalS = linkS + folderS + copyS + saveDiskS
  totalSm,totalSg = b2mg(totalS)
  endTime = datetime.datetime.now()
  totalTime = endTime - startTime
  logging.info("source dir:        %s" %srcDir)
  logging.info("end dir:           %s" %dstDir)
  logging.info("start time:        %s" %str(startTime))
  logging.info("end time:          %s" %str(endTime))
  logging.info("total time:        %s" %str(totalTime))
  logging.info("create link size:  %d(B)=%f(MB)=%f(GB)" %(linkS,linkSm,linkSg))
  logging.info("create folder size:%d(B)=%f(MB)=%f(GB)" %(folderS,folderSm,folderSg))
  logging.info("copy files:        %d(B)=%f(MB)=%f(GB)" %(copyS,copySm,copySg))
  logging.info("save disk size:    %d(B)=%f(MB)=%f(GB)" %(saveDiskS,saveDiskSm,saveDiskg))
  logging.info("total size:        %d(B)=%f(MB)=%f(GB)" %(totalS,totalSm,totalSg))
  logging.info("save disk percent: %s" %(str(saveDiskS/totalS*100)) + '%')

  res = 0
  errors = verifyLog(logFileName)
  if len(errors) > 0: res = 1

  if res == 0:
    logging.info("successful")
  else:
    logging.info("Failed")
  logging.info("=========================================")

  return res

#-----------------------------------------------
# Main entry
if __name__ == "__main__" :
  parser = OptionParser(usage())
  parser.add_option("-R", "--repository", dest="repositoryRoot", \
                    help="Repository root directory")
  parser.add_option("-s", "--source", dest="srcDir", \
                    help="source directory")
  parser.add_option("-d", "--dest", dest="dstDir", \
                    help="dest directory")
  parser.add_option("-a", "--hash", dest="hash", \
                    help="hash type, default will be md5, it also can be sha1,sha256")
  parser.add_option("-v", "--verify", dest="verify",action="store_true", \
                    help="verify after copying, default is false")
  parser.add_option("-l", "--log", dest="logFileName", \
                    help="log file name, default is sourcebasename_pid.log")
  parser.add_option("-n", "--notify", dest="notify", \
                    help="notify email, the valude is recipients, the first one will be sender")
    
  (option,args) = parser.parse_args()
  repositoryRoot = option.repositoryRoot 
  srcDir = option.srcDir 
  dstDir = option.dstDir 
  verify = option.verify
  hash = option.hash
  logFileName = option.logFileName
  notify = option.notify

  if repositoryRoot ==None or repositoryRoot =='' \
     or srcDir == None or srcDir == '' \
     or dstDir == None or dstDir =='':
    sys.stderr.write("must input the options: --repository --source --dest\n") 
    exit(1)
  
  if verify == None: verify = False

  if hash == None or hash == '': hash='md5'

  if logFileName == None or logFileName == '':
    logFileName = os.path.basename(srcDir) + '_' + str(os.getpid()) + '.log'

  if notify != None and len(notify.split(',')) > 1:
     sender = notify[0]
     recipients = str(notify)

  log(logFileName)

  if not os.path.exists(srcDir) or not os.path.isdir(srcDir): 
    logging.error(srcDir + " is not exist or is not one directory")
    exit(1)
    
  if not os.path.exists(repositoryRoot): os.makedirs(repositoryRoot)
  if os.path.exists(dstDir) and os.path.islink(dstDir): os.unlink(dstDir)
  if os.path.exists(dstDir): shutil.rmtree(dstDir)

  startTime = datetime.datetime.now()

  res = 0
  res = main(repositoryRoot,srcDir,dstDir,hash,logFileName)

  if verify:
    res = verifyAll(srcDir,dstDir,logFileName)

  if res : sendMail(logFileName,notify)
  exit(res)
    
    
#python deduplication.py --repository /home/AAA/Repository/ --source /home/AAA/src --dest /home/AAA/dst
